package com.hunter.jobs;

import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import com.database.bean.WebsitsJob;
import com.hunter.tools.ContentProcess;

public class WebPageHunter implements PageProcessor {	
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
	private WebsitsJob wsj;
	
	public WebPageHunter(WebsitsJob wsj)
	{
		this.wsj=wsj;
		this.site.setCharset(wsj.getContent_encode());
	}
	
	@Override
	public Site getSite() {
		return site;
	}

	@Override
	public void process(Page page) {
		System.out.println(wsj.getList_anchor());

		List<String> links = page.getHtml().links().regex(wsj.getList_anchor()).all();
        page.addTargetRequests(links);
        String title=page.getHtml().xpath(wsj.getTitletag()+"/text()").toString();
        String content=page.getHtml().xpath(wsj.getContenttag()+"/html()").toString();
        if(content!=null)
        {
        	ContentProcess cp=new ContentProcess(content);
        	content=cp.getContent();
        }
        page.putField("title", title);
        //page.putField("content", content);
	}

}
